#!/bin/bash

#########################################################################################################
# 需要安装 xvfb 安装命令
# sudo apt-get update
# sudo apt install xvfb
# apt list | grep xvfb

# 关于log不全的问题:需要在runtimedokcer中  /var/log/dlt中  dlt_logstorage.conf里FILTER4的NOFiles由2改到200
########################################################################################################

#ftp info
WORK_PATH=$(readlink -f ${0} | xargs dirname)
script_name=$(basename $0)
source ${WORK_PATH}/.${script_name%\.*}.conf
USER_ID=$(id -u)
DATE=$(date +%Y%m%d-%H%M%S)
start_time_s=$(date +%s)
#variable for sdk
ENV_PATH="${WORK_PATH}/env"
DOCKER_TARGET_PATH="${WORK_PATH}/env/arene-ivi-misc-configs/sdk-hal-docker"
DOCKER_SCRIPT_PATH="${WORK_PATH}/env/arene-ivi-misc-configs"
DOCKER_SCRIPT="${DOCKER_SCRIPT_PATH}/arene-ivi_docker.sh"
DEFAULT_POKY_CONTAINER="arene/poky-songm:ubuntu-20.04"

arene_ivi_misc_config_tar="arene-ivi-misc-configs.tar.gz"
RUNTIME_DOCKER_CONTAINER=sdk-hal-docker_arene_1
HAL_DOCKER_CONTAINER=sdk-hal-docker_app_1
RUNTIME_DOCKER_IMAGE=arene-os-services-cockpit-docker.artifactory-ha.tmc-stargate.com/sdk-docker-runtime-dbg
RELEASE_RUNTIME_DOCKER_REGISTRY=arene-os-services-cockpit-docker-local-tmc-wa-aubass.artifactory-ha.tmc-stargate.com
RELEASE_HAL_DOCKER_REGISTRY=arene-os-services-cockpit-docker-local-tmc-wa.artifactory-ha.tmc-stargate.com
DOCKER_IMAGE_RUNTIME_NAME_RELEASE=sdk-docker-runtime-rel
YOCTO_BUILD_DATE="yocto-build-${DATE}"
YOCRTO_DOCKER_PATH="/home/wovendev/work/yocto-build-$(date +%Y%m%d)"

LOG_PATH="${WORK_PATH}/log"
BUILD_PATH="${WORK_PATH}/src"
RELEASE_DOCKER_PATH="${WORK_PATH}/release_docker"
DOCKER_SCRIPT_COMMAND="${WORK_PATH}/env/arene-ivi-misc-configs/arene-ivi_docker.sh"
log="${LOG_PATH}/${YOCTO_BUILD_DATE}/${script_name%\.*}_${DATE}.log"
########################################################################################################################
ARENE_CONFIG_URL="https://github.tmc-stargate.com/arene-os-services-cockpit-tmc-wa/arene-ivi-misc-configs"
ARENE_CONFIG_REPO="arene-ivi-misc-configs"
ARENE_CONFIG_BRANCH="master"
env_log="${LOG_PATH}/${script_name%\.*}.log"
YOCTO_URL="wovendev@172.30.10.5:"
user_password="tagldev11!!"
########################################################################################################################

function log()
{
    echo "----------------------------------------------------------------------------------------------------------------"
    echo ${1}
    echo "----------------------------------------------------------------------------------------------------------------"
}

function common_check()
{
    local Xvfb_info=$(ps -ef | grep [X]vfb)
    if [[ ${Xvfb_info} ]]; then
        log "[info] Xvfb is exist info = [${Xvfb_info}]"
    else
        log "[info] sh = [Xvfb :1 -screen 0 1280x720x24 &]"
        Xvfb :1 -screen 0 1280x720x24 &
        sleep 30
    fi
}

function start_env
{
    if [[ ! -e ${LOG_PATH} ]]; then
        mkdir -p ${LOG_PATH}
    fi

    if [ -e ${ENV_PATH} ]; then
        log "[NORMAL] ----[${ENV_PATH}] existed !!!----"
    else
        mkdir ${ENV_PATH}
        log "[NORMAL] --create ${ENV_PATH} OK!"
    fi

    find ${WORK_PATH}/log -type d,f -mtime +10 -exec rm -rf {} \;

    if [[ -e ${LOG_PATH}/${DATE} ]]; then
        : > ${env_log}
        echo "[INFO ] ${LOG_PATH}/${DATE} exists." | tee -a ${env_log}
    else
        mkdir -p ${LOG_PATH}/${DATE}
        : > ${env_log}
        echo "[INFO ] mkdir ${LOG_PATH}/${DATE} Success." | tee -a ${env_log}
    fi

    if [[ -e ${RELEASE_DOCKER_PATH} ]]; then
        rm -f ${RELEASE_DOCKER_PATH}/sdk*
    else
        mkdir -p  ${RELEASE_DOCKER_PATH}
    fi 

    common_check

    echo "#####################################################"  | tee -a ${env_log}
    echo "# Start Time     : $(date +%Y/%m/%d-%H:%M:%S)"          | tee -a ${env_log}
    echo "#####################################################"  | tee -a ${env_log}
    echo "" | tee -a ${env_log} 
}

function start_yocto
{
    if [[ ! -e ${LOG_PATH} ]]; then
        mkdir -p ${LOG_PATH}
    fi

    if [[ ! -e ${BUILD_PATH} ]]; then
        mkdir -p ${BUILD_PATH}
    fi

    if [[ -e ${LOG_PATH}/${YOCTO_BUILD_DATE} ]]; then
        : > ${log}
        echo "[INFO ] ${LOG_PATH}/${YOCTO_BUILD_DATE} exists." | tee -a ${log}
    else
        mkdir -p ${LOG_PATH}/${YOCTO_BUILD_DATE}
        mkdir -p ${LOG_PATH}/${YOCTO_BUILD_DATE}/check_result
        mkdir -p ${LOG_PATH}/${YOCTO_BUILD_DATE}/service-start
        : > ${log}
        echo "[INFO ] mkdir ${LOG_PATH}/${YOCTO_BUILD_DATE} Success." | tee -a ${log}
    fi

    common_check

    echo "#####################################################"  | tee -a ${log}
    echo "# Start Time     : $(date +%Y/%m/%d-%H:%M:%S)"          | tee -a ${log}
    echo "#####################################################"  | tee -a ${log}
    echo "" | tee -a ${log} 
}

function succ_exit
{
    local end_time_s=$(date +%s)
    echo "######################################################"  | tee -a ${log}
    echo "# Execution Result : Success "                           | tee -a ${log}
    echo "# Execution Time   : $[ $end_time_s - $start_time_s ]s"  | tee -a ${log}
    echo "# End Time         : $(date +%Y%m%d-%H:%M:%S)"           | tee -a ${log}
    echo "######################################################"  | tee -a ${log}
    exit 0
}
function abnormal_exit
{
    local end_time_s=$(date +%s)
    echo "######################################################"  | tee -a ${log}
    echo "# Execution Result : ERROR !!! "                         | tee -a ${log}
    echo "# Execution Time   : $[ $end_time_s - $start_time_s ]s"  | tee -a ${log}
    echo "# End Time         : $(date +%Y%m%d-%H:%M:%S)"           | tee -a ${log}   
    echo "######################################################"  | tee -a ${log}
    exit 1
}


function get_git_version
{
    pushd $1 >/dev/null
    {
        version=$(git rev-parse HEAD)
        if [[ $? -ne 0 ]]; then 
            echo "[ERR ] line:${LINENO} Failed to get the current version of the specified repo path. "
            abnormal_exit
        else
            echo "${version}"
        fi
    }
    popd >/dev/null
}


function git_clone_update
{
    local git_repo_url=$1
    local git_repo=$2
    local git_branch=$3
    local path=$4
    local git_log="${path}/${git_repo}_git_progress.log"
    :> ${git_log}
    echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
    echo "[INFO] Start update ${git_repo} from github." | tee -a ${git_log}
    if [[ ${git_repo_url} != "" ]] && [[ ${git_repo} != "" ]] && [[ ${git_branch} != "" ]]; then
        if [[ -e ${git_repo} ]] && [[ -e ${git_repo}/.git ]] ;then
            cd ${git_repo}
            # reset branch
            git reset --hard HEAD >> ${git_log}  2>&1
            git clean -df >> ${git_log}  2>&1
            # switch master branch
            if [[ $(git rev-parse --abbrev-ref HEAD) != "master" ]]; then
                git switch master >> ${git_log}  2>&1
            fi
            # delete other local branch
            local other_branches=$(git branch -l |grep -v master)
            if [[ ${other_branches} != "" ]]; then 
                git branch -D ${other_branches} >> ${git_log}  2>&1
            fi
            # sync remote branch name
            git remote prune origin >> ${git_log}  2>&1
            # update all branch
            echo "[INFO] Exec command=>$ git pull" | tee -a ${git_log}
            git pull >> ${git_log}  2>&1
            if [[ $? -ne 0 ]]; then
                echo "[ERR ] Failed to git pull branch." | tee -a ${git_log}
                abnormal_exit
            fi
        else
            # clone repository
            rm -fr ${git_repo}
            echo "[INFO] Exec command=>$ git clone --progress ${git_repo_url} >> ${git_log}  2>&1"
            git clone --progress ${git_repo_url} >> ${git_log}  2>&1
            if [[ $? -ne 0 ]]; then
                echo "[ERR ] Failed to git clone repo. repo_url=>${repo_url}" | tee -a ${git_log}
                abnormal_exit
            fi
            cd ${git_repo}
        fi
        # switch branche
        if [[ $(git rev-parse --abbrev-ref HEAD) != "${git_branch}" ]]; then
            echo "[INFO] Exec command=>$ git switch ${git_branch}" | tee -a ${git_log}
            git switch ${git_branch} >> ${git_log}  2>&1
            if [[ $? -ne 0 ]]; then
                echo "[ERR ] Failed to git switch branch. branch=>${git_branch}" | tee -a ${git_log}
                abnormal_exit
            fi
        fi
        # reset commit id
        if [[ ${git_commitid} != "" ]];then
            echo "[INFO] Exec command=>$ git reset --hard ${git_commitid}" | tee -a ${git_log}
            git reset --hard ${git_commitid} >> ${git_log}  2>&1
            if [[ $? -ne 0 ]]; then
                echo "[ERR ] Failed to git reset --hard ${git_commitid}" | tee -a ${git_log}
                abnormal_exit
            fi
        fi
        cd - >/dev/null
    fi
    echo "[INFO] Successfully update ${git_repo} from github." | tee -a ${git_log}
    echo "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"
    echo ""
}

function install_docker_env() {
    log=${env_log}
    pushd ${ENV_PATH}
    {
        git_clone_update ${ARENE_CONFIG_URL} ${ARENE_CONFIG_REPO} ${ARENE_CONFIG_BRANCH} ${ENV_PATH}
        if [[ $? -ne 0 ]]; then
            echo "[ERR ] line:${LINENO} Failed to download and update test docker config source code. "
            echo "======================================================"
            abnormal_exit
        fi
        get_git_version ${ARENE_CONFIG_REPO}
    }
    popd

    cd ${DOCKER_TARGET_PATH}
    sed -ri 's/(^DOCKER_IMAGE_HAL_NAME=sdk-docker-hal$)/\1-mock/g' .env
    sed -ri 's/(^DOCKER_IMAGE_RUNTIME_NAME=sdk-docker-runtime$)/\1-dbg/g' .env
    sed -ri "s@WORK_DIR=~/hmifw_share@WORK_DIR=${WORK_PATH}@g" .env
    sed -ri "s@DOCKER_PULL_SKIP=0@DOCKER_PULL_SKIP=1@g" .env
    rm -f sdk-docker-*

    if [[ ${1} ]];then
        sed -ri "4s#(DOCKER_IMAGE_HAL_TAG=).*#\1${1}#g" ./.env
        sed -ri "6s#(DOCKER_IMAGE_RUNTIME_TAG=).*#\1${1}#g" ./.env
    fi

    # remove old docker images and containers.
    log "[NORMAL ] docker stop name=sdk-hal-docker*."
    docker stop  $(docker ps -a -q --filter name=sdk-hal-docker*)

    log "[NORMAL ] docker rm name=sdk-hal-docker*."
    docker rm  $(docker ps -a -q --filter name=sdk-hal-docker*)

    log "[NORMAL ] docker rmi -f images=arene-os*/*."
    docker rmi -f $(docker images -q arene-os*/*)

    source .env
    # docker pull runtime
    log "[NORMAL ] docker pull ${DOCKER_REGISTRY}/${DOCKER_IMAGE_RUNTIME_NAME_RELEASE}:${DOCKER_IMAGE_RUNTIME_TAG}"
    docker login ${DOCKER_REGISTRY}
    docker pull ${DOCKER_REGISTRY}/${DOCKER_IMAGE_RUNTIME_NAME_RELEASE}:${DOCKER_IMAGE_RUNTIME_TAG}
    if [[ $? -ne 0 ]]; then
        echo "[ERROR] docker pull ${DOCKER_REGISTRY}/${DOCKER_IMAGE_RUNTIME_NAME_RELEASE}:${DOCKER_IMAGE_RUNTIME_TAG} ERROR !!!"
    fi
    docker logout

    cd ${DOCKER_SCRIPT_PATH}
    sed -i '/^source ${SCRIPT_DIR}.*$/a\# set virtual display \
    Xvfb_info=$(ps -ef | grep Xvfb) \
    if [[ ${Xvfb_info} =~ "Xvfb :1 -screen 0 1280x720x24" ]]; then \
        echo "[info] Xvfb is exist info = [${Xvfb_info}]" \
    else \
        echo "[info] sh = [Xvfb :1 -screen 0 1280x720x24 &]" \
        Xvfb :1 -screen 0 1280x720x24 & \
        sleep 30 \
    fi \
    export DISPLAY=:1.0 \
    export ARENE_IVI_HAL_MODE=MOCK' ${DOCKER_SCRIPT}
    sed -i '/function start_app() {$/a\    sleep 5' ${DOCKER_SCRIPT_PATH}/arene-common_docker.sh
    sync
    
    echo ${user_password} | sudo -S apt -v >/dev/null 2>&1
    export DISPLAY=:1.0
    (${DOCKER_SCRIPT} install 2>&1)
    if [[ $? -ne 0 ]]; then
        echo "[info] sh = [${DOCKER_SCRIPT} install] ERROR !!!"
        abnormal_exit
    fi
}

function install_yocto_docker_env() {
    if [ -e ${DOCKER_TARGET_PATH} ]; then
        log "[NORMAL] ----[${DOCKER_TARGET_PATH}] existed !!!----"
    else
        log "[ERRPR] -- ${DOCKER_TARGET_PATH} does not exist !!!"
        abnormal_exit
    fi
    
    cd ${DOCKER_TARGET_PATH}
    docker_version=$(grep DOCKER_IMAGE_RUNTIME_TAG .env | cut -d = -f 2)

    rm -rf yocto-build*
    rm -f ${LOG_PATH}/${YOCTO_BUILD_DATE}/check_result/*
    
    log "[NORMAL] scp -r ${YOCTO_URL}${YOCRTO_DOCKER_PATH} ./ start"
    scp -r ${YOCTO_URL}${YOCRTO_DOCKER_PATH} ./
    if [[ -e ${YOCRTO_DOCKER_PATH} ]];then
        pushd ${YOCRTO_DOCKER_PATH}
            runtime_file=$(ls arene-image-cockpit-ivi-genericx86-64-*.rootfs.tar.bz2)
            if [[ -e ${runtime_file} ]];then
                log "[NORMAL] yocto build runtime_file = [${runtime_file}]"
            else
                echo -e "\033[31m[ERR ] yocto build runtime_file does not exist !!!"
                abnormal_exit
            fi

            cp *.txt ${LOG_PATH}/${YOCTO_BUILD_DATE}/check_result
            if [[ $? -ne 0 ]]; then
                echo -e "[ERR ] Failed to cp *.txt ${LOG_PATH}/${YOCTO_BUILD_DATE}/check_result !!!"
                abnormal_exit
            fi
        popd    
    else
        echo -e "\033[31m[ERR ] Failed to scp -r wovendev@172.30.10.5:${YOCRTO_DOCKER_PATH} ./ !!!\033[0m"
        abnormal_exit
    fi
    log "[NORMAL] scp wovendev@172.30.10.5:${YOCRTO_DOCKER_PATH} ./ end "

    # remove old docker images and containers.
    log "[NORMAL ] docker stop ${RUNTIME_DOCKER_CONTAINER}"
    docker stop ${RUNTIME_DOCKER_CONTAINER}

    log "[NORMAL ] docker rm ${RUNTIME_DOCKER_CONTAINER}"
    docker rm ${RUNTIME_DOCKER_CONTAINER}

    log "[NORMAL ] docker rmi images=${RUNTIME_DOCKER_IMAGE}"
    docker rmi $(docker images -q ${RUNTIME_DOCKER_IMAGE})

    docker import ${YOCTO_BUILD_DATE}/${runtime_file} ${RUNTIME_DOCKER_IMAGE}:${docker_version}
    if [[ $? -ne 0 ]]; then
        log -e "\033[31m[ERR ] Failed to docker import  ${docker_registry}:${docker_version} !!!\033[0m"
        abnormal_exit
    fi

    echo ${user_password} | sudo -S apt -v >/dev/null 2>&1
    export DISPLAY=:1.0
    (${DOCKER_SCRIPT} restart 2>&1)
    if [[ $? -ne 0 ]]; then
        echo "[ERR ] Failed to restart docker container."
        abnormal_exit
    fi
}

function service_start_check
{
    if [[ -e ${LOG_PATH}/${YOCTO_BUILD_DATE}/service-start ]]; then
        echo "[INFO ] ${LOG_PATH}/${YOCTO_BUILD_DATE}/service-start exists."
        rm -rf ${LOG_PATH}/${YOCTO_BUILD_DATE}/service-start/*
    else
        mkdir -p ${LOG_PATH}/${YOCTO_BUILD_DATE}/service-start
        echo "[INFO ] mkdir ${LOG_PATH}/${YOCTO_BUILD_DATE}/service-start Success."
    fi

    local cfg_docker_registry=$(grep DOCKER_REGISTRY ${DOCKER_TARGET_PATH}/.env | cut -d = -f 2)
    local cfg_docker_image_runtime_name=$(grep DOCKER_IMAGE_RUNTIME_NAME ${DOCKER_TARGET_PATH}/.env | cut -d = -f 2)
    local cfg_docker_image_runtime_tag=$(grep DOCKER_IMAGE_RUNTIME_TAG ${DOCKER_TARGET_PATH}/.env | cut -d = -f 2)
    local cfg_docker_image_hal_name=$(grep DOCKER_IMAGE_HAL_NAME ${DOCKER_TARGET_PATH}/.env | cut -d = -f 2)
    local cfg_docker_image_hal_tag=$(grep DOCKER_IMAGE_HAL_TAG ${DOCKER_TARGET_PATH}/.env | cut -d = -f 2)
    local curr_image_runtime_id=$(docker images ${cfg_docker_registry}/${cfg_docker_image_runtime_name}:${cfg_docker_image_runtime_tag} --format "{{.ID}}")
    local curr_image_hal_id=$(docker images ${cfg_docker_registry}/${cfg_docker_image_hal_name}:${cfg_docker_image_hal_tag} --format "{{.ID}}")
    
    local curr_runtime_id=$(docker ps -a -q --filter name=${RUNTIME_DOCKER_CONTAINER})
    local curr_hal_id=$(docker ps -a -q --filter name=${HAL_DOCKER_CONTAINER})

    local service_check_log="${LOG_PATH}/${YOCTO_BUILD_DATE}/service-start/service_check.log"

    :>${service_check_log}
    echo "--------------- docker images info --------------" | tee -a ${service_check_log}
    echo "RUNTIME DOCKER IMAGE ID = [${curr_image_runtime_id}]" | tee -a ${service_check_log}
    echo "HAL MOCK DOCKER IMAGE ID = [${curr_image_hal_id}]" | tee -a ${service_check_log}
    echo "RUNTIME DOCKER CONTAINER ID = [${curr_runtime_id}]" | tee -a ${service_check_log}
    echo "HAL MOCK DOCKER CONTAINER ID = [${curr_hal_id}]" | tee -a ${service_check_log}
    echo ""
    echo "--------------- check service status and log(start) --------------" | tee -a ${service_check_log}
    for service_file in ${SERVICE_FILE_LIST[@]}; do
        service_status=$(docker exec -it ${RUNTIME_DOCKER_CONTAINER} bash -c "systemctl is-active  ${service_file} | tr -d '\n' " 2>&1)
        service_process_name=$(docker exec -it ${RUNTIME_DOCKER_CONTAINER} bash -c " ps axhw -o cmd | grep -E '^/opt/arene/bin/${service_file}' |grep -v grep | tr -d '\n' " 2>&1)
        service_log="${WORK_PATH}/${service_file}.log"
        # docker exec -it ${runtime_docker_name} bash -c "systemctl status ${service_file} | tail " >${service_log} 2>&1
        docker exec -it ${RUNTIME_DOCKER_CONTAINER} bash -c " journalctl -u ${service_file} | tail " >${service_log} 2>&1

        if [[ ${service_status} != "active" ]]; then
            # echo -e "\033[31m[NG]          ${service_file}        ${service_process_name}:(process not started)\033[0m" 
            echo "[NG]          ${service_file}        ${service_process_name}:(process not started)"                    >> ${service_check_log}
            echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>"                                                                          >> ${service_check_log}
            cat ${service_log}                                                                                           >> ${service_check_log}
            echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>"                                                                          >> ${service_check_log}
        else
            if grep -iE "\b(err|error|failed)\b" ${service_log} >/dev/null ; then
                # echo -e "\033[31m[OK]          ${service_file}        ${service_process_name} :(log error)\033[0m" 
                echo "[OK]          ${service_file}        ${service_process_name} :(log error)"                    >> ${service_check_log}
                echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>"                                                                 >> ${service_check_log}
                cat ${service_log}                                                                                  >> ${service_check_log}
                echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>"                                                                 >> ${service_check_log} 
            else 
                # echo -e "\033[32m[OK]          ${service_file}        ${service_process_name}\033[0m"               
                echo  "[OK]          ${service_file}        ${service_process_name}"                                >> ${service_check_log}         
            fi
        fi
        rm -fr ${service_log} 
    done
    echo "--------------- check service status and log(end) --------------" | tee -a ${service_check_log}
    echo "" | tee -a ${service_check_log}

    echo "--------------- service_start_dlt start ---------------"
    docker exec -it ${RUNTIME_DOCKER_CONTAINER} bash -c "dlt-logstorage-ctrl -s /var/log/dlt;\
                                                    cd /home/root/share/log/${YOCTO_BUILD_DATE}/service-start;\
                                                    rm -rf dlt;\
                                                    cp -r /var/log/dlt/ .;\
                                                    chown -R ${USER_ID}:${USER_ID} dlt ;\
                                                    ps -ef > system_process.txt;\
                                                    journalctl > service_start.log;\
                                                    chown -R ${USER_ID}:${USER_ID} service_start.log ;\
                                                    chown -R ${USER_ID}:${USER_ID} system_process.txt ;\
                                                    exit 0"
    if [[ $? -ne 0 ]]; then
        echo "[ERR ] Failed to service_start_dlt !!!."
        abnormal_exit
    fi
    if [[ -e ${LOG_PATH}/${YOCTO_BUILD_DATE}/service-start/dlt ]]; then
        echo "[INFO ] service_start_dlt Success."
    else
        echo "[ERR ] Failed to service_start_dlt !!!."
        abnormal_exit
    fi
    echo "--------------- service_start_dlt end ---------------"
    echo ""
}

function docker_restart
{
    echo ">>>>>>>>>>> docker_restart start >>>>>>>>>>>>"
    echo "[INFO] Restart docker container."
    if [[ ${XDG_RUNTIME_DIR} == "" ]];then
		echo "[INFO ] export XDG_RUNTIME_DIR= ["
		export XDG_RUNTIME_DIR=/run/user/${USER_ID}
		echo ${XDG_RUNTIME_DIR}
		echo "]"
	fi
    echo ${user_password} | sudo -S apt -v >/dev/null 2>&1
    export DISPLAY=:1.0
    (${DOCKER_SCRIPT_COMMAND} restart 2>&1)
    if [[ $? -ne 0 ]]; then
        echo "[ERR ] Failed to restart docker container."
        abnormal_exit
    fi
    echo "[INFO] Successfully restart docker container."
    sleep 10
    echo "<<<<<<<<<<< docker_restart end <<<<<<<<<<<<<"
    echo ""
}

function yocto_halmock_build
{
    echo ">>>>>>>>>>>>>>>>>>>>>>> update source and build Start >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
    echo "[INFO] Start update source code from github."
    if [[ ! -e ${BUILD_PATH} ]]; then
        mkdir -p ${BUILD_PATH}
    fi
    cd ${BUILD_PATH}
    for git_repo_cfg in ${GIT_SRC_REPO[@]}; do
        local tmp_array=(${git_repo_cfg//:/ })
        local git_group=${tmp_array[0]}
        local git_repo=${tmp_array[1]}
        local git_branch=${tmp_array[2]}
        local git_commitid=${tmp_array[3]}
        local repo_build_path=""

        # git clone or update source code
        git_clone_update "${GIT_SRC_URL}/${git_group}/${git_repo}" ${git_repo} ${git_branch} ${git_commitid} ${BUILD_PATH}
        if [[ $? -ne 0 ]]; then
            echo "[ERR ] line:${LINENO} Failed to download and update test docker config source code. "
            echo "======================================================"
            abnormal_exit
        fi

        if [[ ${git_repo} == "vehicleinfo" ]];then
            repo_build_path="vehicleinfo/vehicle_info"
        else
            repo_build_path=${git_repo}
        fi
        echo "[INFO] repo_build_path=[${repo_build_path}] !!!"

        local cpu_num="$(cat /proc/cpuinfo | grep processor | wc -l)"

        local hal_mock_build_log="${LOG_PATH}/${YOCTO_BUILD_DATE}/${git_repo}_mock_build.log"
        : > ${hal_mock_build_log}
        echo "---------------------------------------------------------------------------------------------------"        >> ${hal_mock_build_log}
        echo "[INFO] Start build and install the mock code of the ${git_repo}."    | tee -a ${hal_mock_build_log}
        docker exec -it ${HAL_DOCKER_CONTAINER}  bash -c "cd /home/user/work/src/${repo_build_path} ; \
                                        rm -rf build_hal ;\
                                        mkdir build_hal ;\
                                        cd build_hal ;\
                                        echo '$ cmake -DCMAKE_DOCKER_TYPE=Hal ${BUILD_OPTION} ..' ; \
                                        cmake  -DCMAKE_DOCKER_TYPE=Hal ${BUILD_OPTION} ..  ; \
                                        echo '$ make -j ${cpu_num}' ; \
                                        make -j ${cpu_num} 2>&1 | tee ; \
                                        make install 2>&1 | tee ; \
                                        make package 2>&1 | tee ; \
                                        chown -R ${USER_ID}:${USER_ID} * ;\
                                        exit 0" >> ${hal_mock_build_log}  2>&1
        if [[ $(grep -Eo "make: \*\*\* [Makefile:[0-9]+: all] Error [0-9]+" ${hal_mock_build_log}) != "" ]]; then
            echo "[WARN] Failed to build and install the mock code of the ${git_repo}."     | tee -a ${hal_mock_build_log}
        fi

        echo "[INFO] Successfully build and install the mock code of the [${git_repo}]."    | tee -a ${hal_mock_build_log}
        echo "-----------------------------------------------------------------------------------------------------"        >> ${hal_mock_build_log}
    done
    echo "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< update source and build end <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"
    echo ""
}

function build_release_test()
{
    echo "[INFO] copy build_release_test application process to runtime docker start."
    docker exec -it ${RUNTIME_DOCKER_CONTAINER} bash -c "cp -r /home/root/share/build_release_test/* /opt/arene/test/ ; exit"
    if [[ $? -ne 0 ]]; then
        echo "[ERR ] Failed to copy build_release_test application process to runtime docker."
        abnormal_exit
    fi
    echo "[INFO] copy build_release_test application process to runtime docker end."
}

function yocto_unit_test ()
{
    echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> [${1}] docker_unit_test start >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"

    rm -rf ${LOG_PATH}/${YOCTO_BUILD_DATE}/${1}_IT*
    local count=1
    while (( ${count} <= ${IT_COUNT} ))
    do
        local it_name=${1}_IT_${count}
        local it_path=${LOG_PATH}/${YOCTO_BUILD_DATE}/${it_name}
        mkdir -p ${it_path}
        for test_unit_name in ${!TEST_TARGET[@]} ; do
            local test_unit_info=${TEST_TARGET[${test_unit_name}]}
            local service_file=$(echo ${test_unit_info} | cut -d : -f 1)
            local test_command=$(echo ${test_unit_info} | cut -d : -f 2)
            local test_log="${LOG_PATH}/${YOCTO_BUILD_DATE}/${it_name}/${test_unit_name}/${test_unit_name}_test_result.log"

            mkdir -p ${it_path}/${test_unit_name}
            :>${test_log}
            echo "[INFO ] mkdir ${it_path}/${test_unit_name} Success."
            echo ""
            echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> [${test_unit_name}] test start >>>>>>>>>>>>"
            
            if [[ "create" == ${1} ]];then
                echo "[INFO]  docker rm -f ${RUNTIME_DOCKER_CONTAINER} and ${HAL_DOCKER_CONTAINER} start"
                docker stop ${RUNTIME_DOCKER_CONTAINER}
                docker rm -f ${RUNTIME_DOCKER_CONTAINER}
                if [[ "Release" == ${2} ]];then
                    docker stop ${HAL_DOCKER_CONTAINER}
                    docker rm -f ${HAL_DOCKER_CONTAINER}
                fi
                echo "[INFO]  docker rm -f ${RUNTIME_DOCKER_CONTAINER} and ${HAL_DOCKER_CONTAINER} end "
            else
                # delete invalid contents
                # # delete dlt log
                docker exec -it ${RUNTIME_DOCKER_CONTAINER} bash -c "rm -fr /var/log/dlt/*; sed -ri 's@NOFiles=2@NOFiles=100@g' /etc/dlt_logstorage.conf; exit"
                if [[ $? -ne 0 ]]; then
                    echo "[ERR ] Failed to delete dlt log for unit ${test_unit_name}."
                    abnormal_exit
                fi                            
                echo "[INFO] Successfully delete dlt log for unit ${test_unit_name}."

                echo "[INFO] rm -rf /var/backups/* for unit ${test_unit_name}."
                docker exec -it ${RUNTIME_DOCKER_CONTAINER} bash -c "rm -rf /var/backups/*;rm -rf /var/backups/backup-fs/VehicleSettings/*; exit"
                # delete hal common test log
                docker exec -it ${HAL_DOCKER_CONTAINER} bash -c "rm -fr /usr/local/bin/nohup.out; exit"
                if [[ $? -ne 0 ]]; then
                    echo "[ERR ] Failed to delete hal common test log for unit ${test_unit_name}."
                    abnormal_exit
                fi                            
                echo "[INFO] Successfully delete hal common test log for unit ${test_unit_name}."
            fi
            
            #################################################################################
            # restart docker container
            #################################################################################
            docker_restart

            #################################################################################
            # service_start_check
            #################################################################################
            service_start_check
            mv ${LOG_PATH}/${YOCTO_BUILD_DATE}/service-start ${it_path}/${test_unit_name}
            #################################################################################
            # start service
            #################################################################################
            local service_status=$(docker exec -it ${RUNTIME_DOCKER_CONTAINER} bash -c "systemctl is-active  ${service_file} | tr -d '\n' " 2>&1)
            if [[ ${service_status} != "active" ]]; then
                echo "[WARR] The ${test_unit_name} service is not started!"
                echo "[INFO] Restart ${test_unit_name} service ...     "
                docker exec -it ${RUNTIME_DOCKER_CONTAINER}  bash -c "systemctl restart ${service_file}" 2>&1
                if [[ $? -ne 0 ]]; then
                    echo "[ERR ] Failed to restart ${test_unit_name} service."
                    abnormal_exit
                fi
                echo "[INFO] Successfully restart ${test_unit_name} service."
            fi

            #################################################################################
            if [[ "test" == ${3} ]];then
                build_release_test
            fi
            
            #################################################################################
            # start execute test
            #################################################################################
            echo "[INFO] Start execute the test for unit ${test_unit_name}."
            echo "$ ${test_command}" > ${test_log}
            docker exec -it ${RUNTIME_DOCKER_CONTAINER} bash -c "${test_command} 2>&1 | tee ;exit" >> ${test_log} 2>&1
            if [[ $? -ne 0 ]]; then
                echo "Failed to execute the test for unit ${test_unit_name}."  >> ${test_log}
                echo "[ERR ] Failed to execute the test for unit ${test_unit_name}."
                abnormal_exit
            fi
            echo "Successfully execute the test for unit ${test_unit_name}."   >> ${test_log}
            echo "[INFO] Successfully execute the test for unit ${test_unit_name}."

            #################################################################################
            # copy test result from runtime docker
            #################################################################################
            echo "[INFO] Start copy the test result for unit ${test_unit_name}."
            docker exec -it ${RUNTIME_DOCKER_CONTAINER} bash -c "dlt-logstorage-ctrl -s /var/log/dlt;\
                                                    gzip -dk /var/log/dlt/*.gz; \
                                                    find /var/log/dlt/ -name "journal.*.dlt" | xargs dlt-convert -a >> /var/log/dlt/journal_dlt_convert.log; \
                                                    cp -rf /tmp/${test_unit_name}_* /home/root/share/log/${YOCTO_BUILD_DATE}/${it_name}/${test_unit_name}/ ;\
                                                    cp -rf /var/log/dlt/ /home/root/share/log/${YOCTO_BUILD_DATE}/${it_name}/${test_unit_name}/ ;\
                                                    chown -R ${USER_ID}:${USER_ID} /home/root/share/log/${YOCTO_BUILD_DATE}/${it_name}/${test_unit_name}/ ;\
                                                    cd /home/root/share/log/${YOCTO_BUILD_DATE}/${it_name}/${test_unit_name}/ ;\
                                                    ps -ef > system_process.txt ;\
                                                    journalctl > service_start.log ;\
                                                    chown -R ${USER_ID}:${USER_ID} system_process.txt ;\
                                                    chown -R ${USER_ID}:${USER_ID} service_start.log ;\
                                                    exit 0"
            if [[ $? -ne 0 ]]; then
                echo "[ERR ] Failed to copy the test result for unit ${test_unit_name}."
                abnormal_exit
            fi
            echo "[INFO] Successfully copy the test result for unit ${test_unit_name}."

            #################################################################################
            # copy hal common log from hal docker
            #################################################################################
            echo "[INFO] Start copy the hal common test log for unit ${test_unit_name}."
            docker exec -it ${HAL_DOCKER_CONTAINER} bash -c "cp -rf /usr/local/bin/nohup.out /home/user/work/log/${YOCTO_BUILD_DATE}/${it_name}/${test_unit_name}/ ;\
                                                    chown -R ${USER_ID}:${USER_ID} /home/user/work/log/${YOCTO_BUILD_DATE}/${it_name}/${test_unit_name}/* ;\
                                                    exit"
            if [[ $? -ne 0 ]]; then
                echo "[ERR ] Failed to copy the hal common test log for unit [${test_unit_name}]."
                abnormal_exit
            fi                            
            echo "[INFO] Successfully copy the hal common test log for unit [${test_unit_name}]."

            #################################################################################
            echo "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< [${test_unit_name}] test end <<<<<<<<<<<<"
            echo ""
            echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>> [${test_unit_name}] test result processing start >>>>>>>>>>>>"
            local unit_service_list=(${TEST_TARGET_SERVICE[${test_unit_name}]})
            local test_result_file="${it_path}/${test_unit_name}/${test_unit_name}_aggregated_result.txt"
            : > ${test_result_file}

            echo "[INFO] unit [${test_unit_name}] Check before test." >> ${test_result_file}
            for service in ${unit_service_list[@]}; do
                grep ${service} ${it_path}/${test_unit_name}/service-start/system_process.txt >> ${test_result_file}
            done
            echo "-------------------------------------------" | tee -a ${test_result_file}
            echo "" | tee -a ${test_result_file}

            echo "[INFO] grep "AllTests" *.xml check test results ." | tee -a ${test_result_file}
            grep "AllTests" ${it_path}/${test_unit_name}/*.xml | tee -a ${test_result_file}
            echo "-------------------------------------------" | tee -a ${test_result_file}
            echo "" | tee -a ${test_result_file}

            echo "[INFO] grep "\[  FAILED  \]" ${test_log} check errors case." | tee -a ${test_result_file}
            grep "\[  FAILED  \]" ${test_log} | tee -a ${test_result_file}
            echo "-------------------------------------------" | tee -a ${test_result_file}
            echo "" | tee -a ${test_result_file}

            echo "[INFO] grep "AddressSanitizer/LeakSanitizer" ${test_log} ." | tee -a ${test_result_file}
            grep "AddressSanitizer" ${test_log} | tee -a ${test_result_file}
            grep "LeakSanitizer" ${test_log} | tee -a ${test_result_file}
            echo "-------------------------------------------" | tee -a ${test_result_file}
            echo "" | tee -a ${test_result_file}

            echo "[INFO] grep "AddressSanitizer/LeakSanitizer" ${it_path}/${test_unit_name}/dlt/journal_dlt_convert.log ." | tee -a ${test_result_file}
            grep "AddressSanitizer" ${it_path}/${test_unit_name}/dlt/journal_dlt_convert.log | tee -a ${test_result_file}
            grep "LeakSanitizer" ${it_path}/${test_unit_name}/dlt/journal_dlt_convert.log | tee -a ${test_result_file}
            echo "-------------------------------------------" | tee -a ${test_result_file}
            echo "" | tee -a ${test_result_file}
            
            echo "[INFO] unit [${test_unit_name}] Check before test." >> ${test_result_file}
            for service in ${unit_service_list[@]}; do
                grep ${service} ${it_path}/${test_unit_name}/system_process.txt >> ${test_result_file}
            done
            echo "-------------------------------------------" >> ${test_result_file}
            echo "" >> ${test_result_file}
            echo "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< [${test_unit_name}] test result processing  end  <<<<<<<<<<<<"
        done
        let "count++"
    done
    echo "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< [${1}] docker_unit_test end <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"
    echo ""
}

########################################################
# main
########################################################
while :
do
    echo "#####################################################################"
    echo "#         Please select one of the options below to proceed         #"
    echo "# Options:                                                          #"
    echo "#         1   DOCKER install                                        #"
    echo "#         2   DOCKER install ver                                    #"
    echo "#         3   YOCTO DOCKER install and test                         #"
    echo "#         4   YOCTO unit test                                       #"
    echo "#         5   Release unit test                                     #"
    echo "#         6   build_release_test unit test                          #"
    echo "#         7   docker restart                                        #"
    echo "#         0   exit                                                  #"
    echo "#####################################################################"

    read -p "Please enter your options:" options
    echo ""
    
    case ${options} in
        1)
            log "[NORMAL] >>>>>>Install docker Start !<<<<<<"
            start_env
            install_docker_env  | tee -a ${env_log}
            log "[NORMAL] >>>>>>Install docker  End  !<<<<<<"
            ;;
        2)
            read -p "Please enter your docker version(eg:v7.1.0):" ver
            log "[NORMAL] >>>>>>Install docker ${ver} Start !<<<<<<"
            start_env
            install_docker_env ${ver}  | tee -a ${env_log}
            log "[NORMAL] >>>>>>Install docker ${ver} End  !<<<<<<"
            ;;
        3)
            log "[NORMAL] >>>>>> YOCTO DOCKER install Start !<<<<<<"
            start_yocto
            install_yocto_docker_env | tee -a ${log}
            sleep 200 
            service_start_check | tee -a ${log}
            rm -rf ${LOG_PATH}/${YOCTO_BUILD_DATE}/service_start_check
            mv ${LOG_PATH}/${YOCTO_BUILD_DATE}/service-start ${LOG_PATH}/${YOCTO_BUILD_DATE}/service_start_check | tee -a ${log}
            yocto_halmock_build | tee -a ${log}
            yocto_unit_test "create" | tee -a ${log}
            # yocto_unit_test "restart" | tee -a ${log}
            succ_exit
            log "[NORMAL] >>>>>> YOCTO DOCKER install End  !<<<<<<"
            ;;
        4)
            log "[NORMAL] >>>>>> YOCTO unit test create Start !<<<<<<"
            start_yocto
            yocto_halmock_build | tee -a ${log}
            yocto_unit_test "create" | tee -a ${log}
            # yocto_unit_test "restart" | tee -a ${log}
            succ_exit
            log "[NORMAL] >>>>>> YOCTO unit test create End  !<<<<<<"
            ;;
        5)
            log "[NORMAL] >>>>>> Release unit test Start !<<<<<<"
            start_yocto
            # yocto_unit_test "restart" | tee -a ${log}
            yocto_unit_test "create" "Release" | tee -a ${log}
            succ_exit
            log "[NORMAL] >>>>>> Release unit test End  !<<<<<<"
            ;;
        6)
            log "[NORMAL] >>>>>> build_release_test unit test Start !<<<<<<"
            start_yocto
            yocto_unit_test "create" "Release" "test" | tee -a ${log}
            build_release_test
            yocto_unit_test "restart" | tee -a ${log}
            succ_exit
            log "[NORMAL] >>>>>> build_release_test unit test End  !<<<<<<"
            ;;
        7)
            log "[NORMAL] >>>>>> docker_restart Start !<<<<<<"
            docker_restart
            log "[NORMAL] >>>>>> docker_restart End  !<<<<<<"
            ;;
        0)
            log "[NORMAL] --exit 0--"
            exit 0
            ;;
        *)
            log "[ERROR ] ------wrong option = [ ${options} ]!!!------"
            ;;
    esac
done
